﻿// webtake.cpp : 定义控制台应用程序的入口点。
//


#include "stdafx.h"
#include <iostream>
#include <fstream>
#include <string>
#include "Common\Include\WinHttpClient.h"
#include <codecvt>
#include <windows.h>
//#include <Wininet.h>
#include "curl\curl.h"
#include "httpclient.h"
#include "Libcurl.h"

using namespace std;

#pragma comment(lib,"ws2_32.lib")
#pragma comment(lib,"Crypt32.lib")
#pragma comment(lib,"Wldap32.lib")


bool IsTextUTF8CSharp(const std::string& inputStream)
{
	int encodingBytesCount = 0;
	bool allTextsAreASCIIChars = true;

	typedef unsigned char byte;

	for (int i = 0; i < inputStream.length(); i++)
	{
		byte current = inputStream[i];

		if ((current & 0x80) == 0x80)
		{
			allTextsAreASCIIChars = false;
		}
		// First byte
		if (encodingBytesCount == 0)
		{
			if ((current & 0x80) == 0)
			{
				// ASCII chars, from 0x00-0x7F
				continue;
			}

			if ((current & 0xC0) == 0xC0)
			{
				encodingBytesCount = 1;
				current <<= 2;

				// More than two bytes used to encoding a unicode char.
				// Calculate the real length.
				while ((current & 0x80) == 0x80)
				{
					current <<= 1;
					encodingBytesCount++;
				}
			}
			else
			{
				// Invalid bits structure for UTF8 encoding rule.
				return false;
			}
		}
		else
		{
			// Following bytes, must start with 10.
			if ((current & 0xC0) == 0x80)
			{
				encodingBytesCount--;
			}
			else
			{
				// Invalid bits structure for UTF8 encoding rule.
				return false;
			}
		}
	}

	if (encodingBytesCount != 0)
	{
		// Invalid bits structure for UTF8 encoding rule.
		// Wrong following bytes count.
		return false;
	}

	// Although UTF8 supports encoding for ASCII chars, we regard as a input stream, whose contents are all ASCII as default encoding.
	return !allTextsAreASCIIChars;
}

//gbk ת utf8
std::string GBKToUTF8(const std::string& strGBK)
{
	std::string strOutUTF8 = "";
	WCHAR * str1;
	int n = MultiByteToWideChar(CP_ACP, 0, strGBK.c_str(), -1, NULL, 0);
	str1 = new WCHAR[n];
	MultiByteToWideChar(CP_ACP, 0, strGBK.c_str(), -1, str1, n);
	n = WideCharToMultiByte(CP_UTF8, 0, str1, -1, NULL, 0, NULL, NULL);
	char * str2 = new char[n];
	WideCharToMultiByte(CP_UTF8, 0, str1, -1, str2, n, NULL, NULL);
	strOutUTF8 = str2;
	delete[]str1;
	str1 = NULL;
	delete[]str2;
	str2 = NULL;
	return strOutUTF8;
}

//utf-8 ת gbk
std::string UTF8ToGBK(const std::string& strUTF8)
{
	int len = MultiByteToWideChar(CP_UTF8, 0, strUTF8.c_str(), -1, NULL, 0);
	unsigned short * wszGBK = new unsigned short[len + 1];
	memset(wszGBK, 0, len * 2 + 2);
	MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)strUTF8.c_str(), -1, (LPWSTR)wszGBK, len);

	len = WideCharToMultiByte(CP_ACP, 0, (LPWSTR)wszGBK, -1, NULL, 0, NULL, NULL);
	char *szGBK = new char[len + 1];
	memset(szGBK, 0, len + 1);
	WideCharToMultiByte(CP_ACP, 0, (LPWSTR)wszGBK, -1, (LPSTR)szGBK, len, NULL, NULL);
	//strUTF8 = szGBK;
	std::string strTemp(szGBK);
	delete[]szGBK;
	delete[]wszGBK;
	return strTemp;
}

// Progress - finished percentage.

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	Progress proc. </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <param name="progress">	The progress. </param>
///
/// <returns>	true if it succeeds, false if it fails. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////

bool ProgressProc(double progress)
{
	//wprintf(L"Current progress: %-.1f%%\r\n", progress);
	printf("Current progress: %-.1f%%\r\n", progress);
	//fflush(stdout);
	return true;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	String to w string. </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <param name="str">	The string. </param>
///
/// <returns>	A std::wstring. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////

std::wstring StringToWString(const std::string& str) {
	int num = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, NULL, 0);
	wchar_t *wide = new wchar_t[num];
	MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, wide, num);
	std::wstring w_str(wide);
	delete[] wide;
	return w_str;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	Gets web content. </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <param name="url">		  	URL of the document. </param>
/// <param name="outFileName">	Filename of the out file. </param>
////////////////////////////////////////////////////////////////////////////////////////////////////

void getWebContent(string url, string outFileName)
{
	wstring wurl = StringToWString(url);
	WinHttpClient client(wurl);
	client.SendHttpRequest();
	//wstring httpResponseHeader = client.GetResponseHeader();
	wstring httpResponseContent = client.GetResponseContent();

	printf("size:%d\n", httpResponseContent.length());
	if (httpResponseContent.empty() == false)
	{
		wofstream f;
		locale LocUtf8 = locale(locale(""), new codecvt_utf8<wchar_t>);
		f.imbue(LocUtf8);
		f.open(outFileName);
		f << httpResponseContent << endl;
	}
}



size_t Reply(void * ptr, size_t size, size_t nmemb, void * stream)
{
	string* str = (string*)stream;
	//cout << *str << endl;
	(*str).append((char*)ptr, size*nmemb);
	return size*nmemb;
}
wstring str_cvt(const string& from_str, int cvt_type) {
	int wstr_len = MultiByteToWideChar(cvt_type, 0, from_str.c_str(), -1, nullptr, 0);
	wchar_t *wstr = new wchar_t[wstr_len + 1];
	memset(wstr, 0, (wstr_len + 1) * sizeof(wchar_t));
	MultiByteToWideChar(cvt_type, 0, from_str.c_str(), -1, wstr, wstr_len);
	wstring ret_str(wstr); delete[] wstr;
	return ret_str;
}
string str_cvt(const wstring& from_str, int cvt_type) {
	int str_len = WideCharToMultiByte(cvt_type, 0, from_str.c_str(), -1, nullptr, 0, nullptr, nullptr);
	char *str = new char[str_len + 1];
	memset(str, 0, str_len + 1);
	WideCharToMultiByte(cvt_type, 0, from_str.c_str(), -1, str, str_len, nullptr, nullptr);
	string ret_str(str); delete[] str;
	return ret_str;
}

CLibcurl libcurl;

//bool getWebContent2(string url, string outFileName)
//{
//	if (libcurl.Get(url.c_str()))
//	{
//		string pHtml = libcurl.GetRespons();
//		bool isUTF8 = IsTextUTF8CSharp(pHtml);
//		if (!isUTF8)
//			pHtml = GBKToUTF8(pHtml);
//		cout << "size：" << pHtml.length() << endl;
//		ofstream f(outFileName);
//		f << pHtml << endl;
//	}
//	else
//	{
//		return false;
//		cerr << libcurl.GetError() << endl;
//	}
//		
//
//	return true;
//}

bool getWebContent2(string url, string outFileName)
{
	int reConnectCount = 4;
	while (reConnectCount > 0)
	{
		--reConnectCount;
		if (libcurl.Get(url.c_str()))
		{
			string pHtml = libcurl.GetRespons();
			bool isUTF8 = IsTextUTF8CSharp(pHtml);
			if (!isUTF8)
				pHtml = GBKToUTF8(pHtml);
			cout << "size：" << pHtml.length() << endl;
			ofstream f(outFileName);
			f << pHtml << endl;
			break;
		}
		else
		{
			cerr << libcurl.GetError() << endl;
			if (reConnectCount > 0)
				cerr << "will reconnect!" << endl;
			else return false;
		}
	}

	return true;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	Gets urls content. </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <param name="urlFile">	The URL file. </param>
///
/// <returns>	true if it succeeds, false if it fails. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////

bool getUrlsContent(string urlFile, int flag = 0)
{
	//打开url文件
	ifstream urlf(urlFile);

	if (!urlf.is_open())
	{
		cerr << "URL file OPEN ERROR" << endl;
		return false;
	}
	string url;
	string outName;
	int index = 1;
	while (getline(urlf, url))
	{
		cout << "[" << index << "]" << url << " is opening!" << endl;
		outName = to_string(index) + ".urldata";
		url = "http://" + url;
		if (flag == 0)
			getWebContent(url, outName);
		else getWebContent2(url, "curl" + outName);
		index++;
	}
	return true;
}


string FilterHtmlTagsString(string &strContent)
{
	string ret("");
	bool text_state(true);
	char character;
	for (int i = 0; i < strContent.length(); i++)
	{
		character = strContent[i];
		if (text_state)
		{
			if (character == '<')
				text_state = false;
			else
				ret += character;
		}
		else
		{
			if (character == '>')
				text_state = true;
		}
	}

	return ret;
}

string FilterEnglishString(string &strContent)
{
	string ret("");
	for (int i = 0; i < strContent.length(); i++)
	{
		if ((byte)strContent[i] > 127)
			ret += strContent[i];

	}

	return ret;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	在文件中过滤掉英文字符，直接遍历文件 </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <param name="infile"> 要过滤的文件 </param>
/// <param name="outfile">	过滤后输出文件 </param>
///
/// <returns>	返回值等于0时标识成功，否则失败 </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////
int Filter(string infile, string outfile)
{
	char character;
	bool text_state(true);
	ifstream html(infile);

	if (html.fail())
	{
		cerr << "Error opening input file >>" << infile << endl;
		return 1;
	}

	ofstream htmltext(outfile, ios::app);
	html.get(character);
	while (!html.eof())
	{
		if ((byte)character > 127)
			htmltext << character;
		html.get(character);
	}

	htmltext << endl;
	return 0;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	在文件中过滤掉英文字符，直接遍历字符串</summary>
///
/// <remarks>	Administrator, 2017/7/10. </remarks>
///
/// <param name="infile"> 要过滤的文件 </param>
/// <param name="outfile">	过滤后输出文件 </param>
///
/// <returns>	返回值等于0时标识成功，否则失败 </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////

int Filter1(string infile, string outfile)
{
	char character;
	bool text_state(true);
	ifstream html(infile);

	if (html.fail())
	{
		cerr << "Error opening input file >>" << infile << endl;
		return 1;
	}
	string strfileContent;
	strfileContent.assign(istreambuf_iterator<char>(html), std::istreambuf_iterator<char>());
	strfileContent = FilterEnglishString(strfileContent);
	ofstream htmltext(outfile, ios::app);
	htmltext << strfileContent << endl;

	return 0;
}

int TextPreprocessing(string filePrefix = "", string fileEx = "", int flag = 0)
{
	for (int i = 0; i < 30; i++)
	{
		string filename = filePrefix + to_string(i + 1) + fileEx;
		if (flag == 0)
			Filter(filename, "filterResult.txt");
		else
			Filter1(filename, "filterResult1.txt");
	}

	return 0;
}


////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>	Main entry-point for this application. </summary>
///
/// <remarks>	Administrator, 2017/7/9. </remarks>
///
/// <returns>	Exit-code for the process - 0 for success, else an error code. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////


//int main()
//{
//	string data; // 返回的内容  
//	CURL *curl;
//	CURLcode res;
//
//	curl_global_init(CURL_GLOBAL_ALL);
//
//	curl = curl_easy_init();
//	if (curl) {
//		curl_easy_setopt(curl, CURLOPT_URL, "https://kyfw.12306.cn/otn/login/init");
//		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
//		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
//		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Reply);
//		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
//		res = curl_easy_perform(curl);
//		if (res != CURLE_OK)
//			fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
//		curl_easy_cleanup(curl);
//	}
//	curl_global_cleanup();
//	//cout << str_cvt(str_cvt(data, CP_UTF8), CP_ACP) << endl;
//	ofstream htmltext("curltest.txt");
//	htmltext<< data << endl;
//	system("pause");
//	return 0;
//}

class CLibcurlCallbackEx
	: public CLibcurlCallback
{
public:
	virtual void Progress(void* lpParam, double dTotal, double dLoaded)
	{
		if (dTotal == 0.0)
			return;
		double bPercent = (dLoaded / dTotal) * 100;
		printf("下载进度：%0.2lf%%\n", bPercent);
	}

};


void ClibcurlGetTest()
{
	CLibcurl libcurl;
	libcurl.SetUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36");
	libcurl.Get("http://www.rednet.cn");
	const char* pHtml = libcurl.GetResponsPtr();
	const char* pError = libcurl.GetError();
	string strRet(pHtml);
	ofstream of("libcurlest.txt");
	of << strRet;
}

void httpClientGetTest()
{
	CHttpClient hc;
	string strRet;
	hc.Get("http://www.baidu.com", strRet);
	ofstream of("hctest.txt");
	of << strRet;
}

void winhttpcrulspeedcompare()
{
	//1.循环读url内容，保存文件。
	//2.循环过滤文件，保存到一个文件
	//3. 这种操作对文件io操作比较频繁 
	int start = clock();
	getUrlsContent("news.txt");
	double c1 = double(clock() - start) / CLOCKS_PER_SEC;
	printf("winhttp take time:%.3lf sec\n\n", c1);

	start = clock();
	getUrlsContent("news.txt", 1);
	double c2 = double(clock() - start) / CLOCKS_PER_SEC;
	printf("curl take time:%.3lf sec\n\n", c2);

	printf("minus difference:%.3lf sec\n\n", c1 - c2);
}

void delFileLine(string filename, string strLabel)
{
	//读取文件行内容过滤后保存到字符串数组
	vector<string> vecContent;
	//string strfileContent;
	string strLine;
	ifstream inFile(filename);
	while (getline(inFile, strLine))
	{
		//过滤
		if (strLine.rfind(strLabel) == -1)
			vecContent.push_back(strLine);
	}
	inFile.close();

	ofstream outFile(filename);
	vector<string>::const_iterator iter = vecContent.begin();
	for (; vecContent.end() != iter; ++iter)
	{
		string s = *iter;
		if (vecContent.end() - 1 != iter)
			s += "\n";
		outFile << s;//<< endl;
	}
}

bool delFileLine2(string filename, string strLabel)
{
	//读取文件行内容过滤后保存到字符串数组
	vector<string> vecContent;
	string strLine;
	ifstream inFile(filename);
	if (!inFile.is_open()) return false;
	while (getline(inFile, strLine))
	{
		//过滤
		if (strLine.rfind(strLabel) == -1)
			vecContent.push_back(strLine);
	}
	inFile.close();

	//覆盖保存
	int vc = vecContent.size();
	if (vc > 0)
	{
		ofstream outFile(filename);

		for (int i = 0; i < vc; ++i)
		{
			string s = vecContent[i];
			if (vc - 1 != i)
				s += "\n";
			outFile << s;//<< endl;
		}
	}

	return true;
}

void delFileLine1(string filename, string strLabel)
{
	//读取文件行内容过滤后保存到字符串数组
	//vector<string> vecContent;
	string strfileContent;
	string strLine;
	ifstream inFile(filename);
	if (inFile.is_open())
	{
		while (getline(inFile, strLine))
		{
			//过滤
			if (strLine.rfind(strLabel) == -1)
			{
				strfileContent += strLine;
				strfileContent += "\n";
			}

		}
		inFile.close();

		if (strfileContent.length() > 0)
		{
			//strfileContent = strfileContent.substr(0, strfileContent.length() - 1);
			*(--strfileContent.end()) = '\0';
			ofstream outFile(filename);
			outFile << strfileContent;
		}
	}

}

int main()
{
	//winhttpcrulspeedcompare();

	int start = clock();
	//getWebContent2("http://www.chinanews.com", "curltest1.txt");
	//getWebContent2("http://www.rednet.cn/index.html", "curltest1.txt");
	//TextPreprocessing("curl", ".urldata",1);

	//delFileLine("model_wws.tab", "__label__news");
	delFileLine1("model_wws.tab", "__label__news");
	double c2 = double(clock() - start) / CLOCKS_PER_SEC;
	printf("TextPreprocessing time:%.3lf sec\n\n", c2);

	//Filter("1.urldata", "12.txt");

	system("pause");
	return 0;
}
